{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Format DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(442, 11)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>bmi</th>\n",
       "      <th>bp</th>\n",
       "      <th>s1</th>\n",
       "      <th>s2</th>\n",
       "      <th>s3</th>\n",
       "      <th>s4</th>\n",
       "      <th>s5</th>\n",
       "      <th>s6</th>\n",
       "      <th>progression</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.038076</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.061696</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>-0.044223</td>\n",
       "      <td>-0.034821</td>\n",
       "      <td>-0.043401</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.019908</td>\n",
       "      <td>-0.017646</td>\n",
       "      <td>151.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.001882</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.051474</td>\n",
       "      <td>-0.026328</td>\n",
       "      <td>-0.008449</td>\n",
       "      <td>-0.019163</td>\n",
       "      <td>0.074412</td>\n",
       "      <td>-0.039493</td>\n",
       "      <td>-0.068330</td>\n",
       "      <td>-0.092204</td>\n",
       "      <td>75.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.085299</td>\n",
       "      <td>0.050680</td>\n",
       "      <td>0.044451</td>\n",
       "      <td>-0.005671</td>\n",
       "      <td>-0.045599</td>\n",
       "      <td>-0.034194</td>\n",
       "      <td>-0.032356</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>0.002864</td>\n",
       "      <td>-0.025930</td>\n",
       "      <td>141.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.089063</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.011595</td>\n",
       "      <td>-0.036656</td>\n",
       "      <td>0.012191</td>\n",
       "      <td>0.024991</td>\n",
       "      <td>-0.036038</td>\n",
       "      <td>0.034309</td>\n",
       "      <td>0.022692</td>\n",
       "      <td>-0.009362</td>\n",
       "      <td>206.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.005383</td>\n",
       "      <td>-0.044642</td>\n",
       "      <td>-0.036385</td>\n",
       "      <td>0.021872</td>\n",
       "      <td>0.003935</td>\n",
       "      <td>0.015596</td>\n",
       "      <td>0.008142</td>\n",
       "      <td>-0.002592</td>\n",
       "      <td>-0.031991</td>\n",
       "      <td>-0.046641</td>\n",
       "      <td>135.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        age       sex       bmi        bp        s1        s2        s3  \\\n",
       "0  0.038076  0.050680  0.061696  0.021872 -0.044223 -0.034821 -0.043401   \n",
       "1 -0.001882 -0.044642 -0.051474 -0.026328 -0.008449 -0.019163  0.074412   \n",
       "2  0.085299  0.050680  0.044451 -0.005671 -0.045599 -0.034194 -0.032356   \n",
       "3 -0.089063 -0.044642 -0.011595 -0.036656  0.012191  0.024991 -0.036038   \n",
       "4  0.005383 -0.044642 -0.036385  0.021872  0.003935  0.015596  0.008142   \n",
       "\n",
       "         s4        s5        s6  progression  \n",
       "0 -0.002592  0.019908 -0.017646        151.0  \n",
       "1 -0.039493 -0.068330 -0.092204         75.0  \n",
       "2 -0.002592  0.002864 -0.025930        141.0  \n",
       "3  0.034309  0.022692 -0.009362        206.0  \n",
       "4 -0.002592 -0.031991 -0.046641        135.0  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.datasets import load_diabetes\n",
    "\n",
    "data = load_diabetes()\n",
    "train_df = pd.DataFrame(data=data.data, columns=data.feature_names)\n",
    "train_df[\"progression\"] = data.target\n",
    "\n",
    "print(train_df.shape)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set Up Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-Experiment Key:   '_5zDEjF2PgF4Tj2bxagz_-JU-Nig_vU3WpP_thE_rsk='\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import Environment, CVExperiment\n",
    "\n",
    "env = Environment(\n",
    "    train_dataset=train_df,\n",
    "    results_path=\"HyperparameterHunterAssets\",\n",
    "    target_column=\"progression\",\n",
    "    metrics=[\"mean_absolute_error\"],\n",
    "    cv_type=\"KFold\",\n",
    "    cv_params=dict(n_splits=12, shuffle=True, random_state=32),\n",
    "    runs=2,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that HyperparameterHunter has an active `Environment`, we can do two things:\n",
    "\n",
    "# 1. Perform Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<18:55:50> Validated Environment:  '_5zDEjF2PgF4Tj2bxagz_-JU-Nig_vU3WpP_thE_rsk='\n",
      "<18:55:50> Initialized Experiment: 'c3555fcd-ff02-4a5a-aa7f-f2cbbe63a322'\n",
      "<18:55:50> Hyperparameter Key:     'dJFRA_seUsBfnxhHdFj0t3yMOo_Z_k_mf4HqTTAkhJc='\n",
      "<18:55:50> \n",
      "<18:55:50> F0/R0  |  OOF(mean_absolute_error=50.23672)  |  Time Elapsed: 0.11365 s\n",
      "<18:55:50> F0/R1  |  OOF(mean_absolute_error=50.59791)  |  Time Elapsed: 0.11199 s\n",
      "<18:55:50> F0.0 AVG:   OOF(mean_absolute_error=49.50914)  |  Time Elapsed: 0.22989 s\n",
      "<18:55:50> F1/R0  |  OOF(mean_absolute_error=54.15111)  |  Time Elapsed: 0.11137 s\n",
      "<18:55:50> F1/R1  |  OOF(mean_absolute_error=49.27629)  |  Time Elapsed: 0.11155 s\n",
      "<18:55:50> F0.1 AVG:   OOF(mean_absolute_error=51.17912)  |  Time Elapsed: 0.22752 s\n",
      "<18:55:50> F2/R0  |  OOF(mean_absolute_error=46.85189)  |  Time Elapsed: 0.11181 s\n",
      "<18:55:50> F2/R1  |  OOF(mean_absolute_error=44.18693)  |  Time Elapsed: 0.11129 s\n",
      "<18:55:50> F0.2 AVG:   OOF(mean_absolute_error=44.42631)  |  Time Elapsed: 0.22724 s\n",
      "<18:55:50> F3/R0  |  OOF(mean_absolute_error=46.31492)  |  Time Elapsed: 0.11274 s\n",
      "<18:55:50> F3/R1  |  OOF(mean_absolute_error=50.49965)  |  Time Elapsed: 0.11091 s\n",
      "<18:55:50> F0.3 AVG:   OOF(mean_absolute_error=47.50925)  |  Time Elapsed: 0.22782 s\n",
      "<18:55:51> F4/R0  |  OOF(mean_absolute_error=47.47420)  |  Time Elapsed: 0.11176 s\n",
      "<18:55:51> F4/R1  |  OOF(mean_absolute_error=45.85264)  |  Time Elapsed: 0.11189 s\n",
      "<18:55:51> F0.4 AVG:   OOF(mean_absolute_error=45.62287)  |  Time Elapsed: 0.22794 s\n",
      "<18:55:51> F5/R0  |  OOF(mean_absolute_error=43.88476)  |  Time Elapsed: 0.11159 s\n",
      "<18:55:51> F5/R1  |  OOF(mean_absolute_error=44.77085)  |  Time Elapsed: 0.11162 s\n",
      "<18:55:51> F0.5 AVG:   OOF(mean_absolute_error=44.05622)  |  Time Elapsed: 0.22771 s\n",
      "<18:55:51> F6/R0  |  OOF(mean_absolute_error=54.62479)  |  Time Elapsed: 0.11163 s\n",
      "<18:55:51> F6/R1  |  OOF(mean_absolute_error=50.46297)  |  Time Elapsed: 0.1115 s\n",
      "<18:55:51> F0.6 AVG:   OOF(mean_absolute_error=51.92307)  |  Time Elapsed: 0.22762 s\n",
      "<18:55:51> F7/R0  |  OOF(mean_absolute_error=44.67608)  |  Time Elapsed: 0.11253 s\n",
      "<18:55:51> F7/R1  |  OOF(mean_absolute_error=45.60778)  |  Time Elapsed: 0.11124 s\n",
      "<18:55:51> F0.7 AVG:   OOF(mean_absolute_error=44.34810)  |  Time Elapsed: 0.22781 s\n",
      "<18:55:51> F8/R0  |  OOF(mean_absolute_error=51.71038)  |  Time Elapsed: 0.11179 s\n",
      "<18:55:52> F8/R1  |  OOF(mean_absolute_error=57.73887)  |  Time Elapsed: 0.11316 s\n",
      "<18:55:52> F0.8 AVG:   OOF(mean_absolute_error=53.71742)  |  Time Elapsed: 0.22897 s\n",
      "<18:55:52> F9/R0  |  OOF(mean_absolute_error=59.90224)  |  Time Elapsed: 0.11175 s\n",
      "<18:55:52> F9/R1  |  OOF(mean_absolute_error=57.05790)  |  Time Elapsed: 0.11258 s\n",
      "<18:55:52> F0.9 AVG:   OOF(mean_absolute_error=57.98135)  |  Time Elapsed: 0.22887 s\n",
      "<18:55:52> F10/R0  |  OOF(mean_absolute_error=47.20032)  |  Time Elapsed: 0.11227 s\n",
      "<18:55:52> F10/R1  |  OOF(mean_absolute_error=50.38856)  |  Time Elapsed: 0.11246 s\n",
      "<18:55:52> F0.10 AVG:   OOF(mean_absolute_error=47.91546)  |  Time Elapsed: 0.22915 s\n",
      "<18:55:52> F11/R0  |  OOF(mean_absolute_error=51.39181)  |  Time Elapsed: 0.1118 s\n",
      "<18:55:52> F11/R1  |  OOF(mean_absolute_error=56.31213)  |  Time Elapsed: 0.11204 s\n",
      "<18:55:52> F0.11 AVG:   OOF(mean_absolute_error=53.06098)  |  Time Elapsed: 0.2283 s\n",
      "<18:55:52> \n",
      "<18:55:52> FINAL:    OOF(mean_absolute_error=49.26527)  |  Time Elapsed: 2.75077 s\n",
      "<18:55:52> \n",
      "<18:55:52> Saving results for Experiment: 'c3555fcd-ff02-4a5a-aa7f-f2cbbe63a322'\n"
     ]
    }
   ],
   "source": [
    "from xgboost import XGBRegressor\n",
    "\n",
    "experiment = CVExperiment(\n",
    "    model_initializer=XGBRegressor,\n",
    "    model_init_params=dict(max_depth=4, n_estimators=400, subsample=0.5),\n",
    "    model_extra_params=dict(fit=dict(eval_metric=\"mae\")),\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Hyperparameter Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validated Environment with key: \"_5zDEjF2PgF4Tj2bxagz_-JU-Nig_vU3WpP_thE_rsk=\"\n",
      "\u001b[31mSaved Result Files\u001b[0m\n",
      "\u001b[31m____________________________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   (fit, eval_metric) |   booster |   learning_rate |   max_depth |   n_estimators | \n",
      "Experiments matching cross-experiment key/algorithm: 1\n",
      "Experiments fitting in the given space: 1\n",
      "Experiments matching current guidelines: 1\n",
      "    0 | c3555fcd | 00m00s | \u001b[35m  49.26527\u001b[0m | \u001b[32m                 mae\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.1000\u001b[0m | \u001b[32m          4\u001b[0m | \u001b[32m           400\u001b[0m | \n",
      "\u001b[31mHyperparameter Optimization\u001b[0m\n",
      "\u001b[31m____________________________________________________________________________________________________________________________\u001b[0m\n",
      " Step |       ID |   Time |      Value |   (fit, eval_metric) |   booster |   learning_rate |   max_depth |   n_estimators | \n",
      "    1 | 75272ef8 | 00m00s |   65.67196 |                  mae |  gblinear |          0.1864 |          13 |            188 | \n",
      "    2 | 142fb7d8 | 00m00s |   65.67196 |                  mae |  gblinear |          0.2280 |           5 |            109 | \n",
      "    3 | dbad46a0 | 00m00s |   65.67192 |                 rmse |  gblinear |          0.0195 |           3 |            892 | \n",
      "    4 | 14bf99ed | 00m00s |   65.67196 |                 rmse |  gblinear |          0.3548 |          20 |            285 | \n",
      "    5 | 544c32b9 | 00m00s |   65.67196 |                 rmse |  gblinear |          0.1826 |          13 |            220 | \n",
      "    6 | 39cb7c5e | 00m00s |   65.67196 |                 rmse |  gblinear |          0.2305 |          17 |            722 | \n",
      "    7 | b0c77a00 | 00m00s |   65.67195 |                  mae |  gblinear |          0.0804 |          18 |            329 | \n",
      "    8 | a82077af | 00m12s | \u001b[35m  47.23121\u001b[0m | \u001b[32m                rmse\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.0833\u001b[0m | \u001b[32m         15\u001b[0m | \u001b[32m           599\u001b[0m | \n",
      "    9 | 77db1fdc | 00m00s |   65.67196 |                 rmse |  gblinear |          0.1782 |          20 |            335 | \n",
      "   10 | 5112986b | 00m13s |   47.52421 |                 rmse |    gbtree |          0.0874 |          19 |            659 | \n",
      "   11 | 7815ecf8 | 00m11s | \u001b[35m  46.97827\u001b[0m | \u001b[32m                 mae\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.0838\u001b[0m | \u001b[32m         11\u001b[0m | \u001b[32m           785\u001b[0m | \n",
      "   12 | f4eaf14b | 00m13s |   47.20203 |                 rmse |    gbtree |          0.0319 |          11 |            787 | \n",
      "   13 | 1f7b037c | 00m16s | \u001b[35m  46.86796\u001b[0m | \u001b[32m                 mae\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.0663\u001b[0m | \u001b[32m         19\u001b[0m | \u001b[32m           773\u001b[0m | \n",
      "   14 | 4166058e | 00m17s |   47.24333 |                  mae |    gbtree |          0.0458 |          18 |            778 | \n",
      "   15 | 8a8e12d2 | 00m17s |   46.92410 |                  mae |    gbtree |          0.0627 |          20 |            770 | \n",
      "   16 | d45375ff | 00m18s |   46.91131 |                  mae |    gbtree |          0.0588 |          20 |            800 | \n",
      "   17 | 60cdcd71 | 00m18s |   47.49131 |                 rmse |    gbtree |          0.0624 |          19 |            848 | \n",
      "   18 | 51b6838c | 00m15s | \u001b[35m  46.72596\u001b[0m | \u001b[32m                rmse\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.0720\u001b[0m | \u001b[32m         19\u001b[0m | \u001b[32m           699\u001b[0m | \n",
      "   19 | d072f707 | 00m14s | \u001b[35m  46.46380\u001b[0m | \u001b[32m                 mae\u001b[0m | \u001b[32m   gbtree\u001b[0m | \u001b[32m         0.0729\u001b[0m | \u001b[32m         19\u001b[0m | \u001b[32m           669\u001b[0m | \n",
      "   20 | b428e014 | 00m13s |   46.94371 |                  mae |    gbtree |          0.0752 |          19 |            606 | \n",
      "Optimization loop completed in 0:03:06.131099\n",
      "Best score was 46.463803809153006 from Experiment \"d072f707-4ac0-46b8-ac18-62e5be31f7c0\"\n"
     ]
    }
   ],
   "source": [
    "from hyperparameter_hunter import GBRT, Real, Integer, Categorical\n",
    "\n",
    "optimizer = GBRT(iterations=20, random_state=32)\n",
    "\n",
    "optimizer.forge_experiment(\n",
    "    model_initializer=XGBRegressor,\n",
    "    model_init_params=dict(\n",
    "        max_depth=Integer(2, 20),\n",
    "        n_estimators=Integer(100, 900),\n",
    "        learning_rate=Real(0.0001, 0.5),\n",
    "        subsample=0.5,\n",
    "        booster=Categorical([\"gbtree\", \"gblinear\"]),\n",
    "    ),\n",
    "    model_extra_params=dict(\n",
    "        fit=dict(\n",
    "            eval_metric=Categorical([\"rmse\", \"mae\"]),\n",
    "        ),\n",
    "    ),\n",
    ")\n",
    "\n",
    "optimizer.go()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notice, `optimizer` recognizes our earlier `experiment`'s hyperparameters fit inside the search space/guidelines set for `optimizer`.\n",
    "\n",
    "Then, when optimization is started, it automatically learns from `experiment`'s results - without any extra work for us!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
